from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

# re-check cause this is very important

# The purpose of this modification of the program is to change from add one to add partial
PARTIAL_CONSTANT_point1 = 0.1
PARTIAL_CONSTANT_point2 = 0.2
PARTIAL_CONSTANT_point3 = 0.3
PARTIAL_CONSTANT_point4 = 0.4
PARTIAL_CONSTANT_point5 = 0.5
PARTIAL_CONSTANT_point6 = 0.6
PARTIAL_CONSTANT_point7 = 0.7
PARTIAL_CONSTANT_point8 = 0.8
PARTIAL_CONSTANT_point9 = 0.9

# The sole purpose of this program is to compute the add one probability
NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES = 351734
SIZE_OF_VOCABULARY = 37728619

queryTemrsWithTheirRealFreqIn85KQueriesDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsInQueries_head_85K_0_85%"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    realFreq = int( lineElements[1] )
    # add one version
    # newFreq_point7 = realFreq + 1
    
    # add partial of one version
    newFreq_point1 = realFreq + PARTIAL_CONSTANT_point1
    newFreq_point2 = realFreq + PARTIAL_CONSTANT_point2
    newFreq_point3 = realFreq + PARTIAL_CONSTANT_point3
    newFreq_point4 = realFreq + PARTIAL_CONSTANT_point4
    newFreq_point5 = realFreq + PARTIAL_CONSTANT_point5
    newFreq_point6 = realFreq + PARTIAL_CONSTANT_point6
    newFreq_point7 = realFreq + PARTIAL_CONSTANT_point7
    newFreq_point8 = realFreq + PARTIAL_CONSTANT_point8
    newFreq_point9 = realFreq + PARTIAL_CONSTANT_point9
    
    # add one version
    # addOneEstimatedProbability_point7 = newFreq_point7 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY)
    
    # add partial of one version
    addOneEstimatedProbability_point1 = newFreq_point1 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point1)
    addOneEstimatedProbability_point2 = newFreq_point2 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point2)
    addOneEstimatedProbability_point3 = newFreq_point3 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point3)
    addOneEstimatedProbability_point4 = newFreq_point4 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point4)
    addOneEstimatedProbability_point5 = newFreq_point5 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point5)
    addOneEstimatedProbability_point6 = newFreq_point6 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point6)
    addOneEstimatedProbability_point7 = newFreq_point7 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point7)
    addOneEstimatedProbability_point8 = newFreq_point8 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point8)
    addOneEstimatedProbability_point9 = newFreq_point9 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point9)
    
    # for the unseen words, the new probablity will be:
    # addOneEstimatedProbability_point7 = 1 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY)
        
    if queryTerm not in queryTemrsWithTheirRealFreqIn85KQueriesDict:
        queryTemrsWithTheirRealFreqIn85KQueriesDict[queryTerm] = (realFreq,newFreq_point7,addOneEstimatedProbability_point9,addOneEstimatedProbability_point8,addOneEstimatedProbability_point7,addOneEstimatedProbability_point6,addOneEstimatedProbability_point5,addOneEstimatedProbability_point4,addOneEstimatedProbability_point3,addOneEstimatedProbability_point2,addOneEstimatedProbability_point1)
    else:
        print "Unexpected Behaviour"
        exit(1)
    
    

print "len(queryTemrsWithTheirRealFreqIn85KQueriesDict):",len(queryTemrsWithTheirRealFreqIn85KQueriesDict)
inputFileHandler.close()

'''
# checking mechanism NOT working any more.
# check that the add one method is summing up to 1
currentTotalProbablity = 0.0
for queryTerm in queryTemrsWithTheirRealFreqIn85KQueriesDict:
    (_,_,addOneEstimatedProbability_point7) = queryTemrsWithTheirRealFreqIn85KQueriesDict[queryTerm]
    currentTotalProbablity += addOneEstimatedProbability_point7

# add one version
# currentTotalProbablity = currentTotalProbablity + ( SIZE_OF_VOCABULARY - len(queryTemrsWithTheirRealFreqIn85KQueriesDict) ) * 1 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY)

# add partial of one version
currentTotalProbablity = currentTotalProbablity + ( SIZE_OF_VOCABULARY - len(queryTemrsWithTheirRealFreqIn85KQueriesDict) ) * PARTIAL_CONSTANT_point7 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point7)

print "NEW currentTotalProbablity:",currentTotalProbablity
'''

# logic for adding the probabilities into the file
outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom4KWithTheLatestProbabilitySettings"
outputFileHanlder = open(outputFileName,"w")

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom4KWithTheirTrueProbablityAndPredictedProbablity_withNormalizedPredictedProbabilityAdded_withAddONEUnnormalizedProbabilityAdded_withNormalizedPredictedProbabilityAdded"
inputFileHandler = open(inputFileName,"r")
oldHeadLine = inputFileHandler.readline().strip()
newHeadLine = oldHeadLine + " " + "point9" + " " + "point8" + " " + "point7" + " " + "point6" + " " + "point5" + " " + "point4" + " " + "point3" + " " + "point2" + " " + "point1" + "\n"
outputFileHanlder.write(newHeadLine)

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    addOneEstimatedProbability_point9 = 0.0
    addOneEstimatedProbability_point8 = 0.0
    addOneEstimatedProbability_point7 = 0.0
    addOneEstimatedProbability_point6 = 0.0
    addOneEstimatedProbability_point5 = 0.0
    addOneEstimatedProbability_point4 = 0.0
    addOneEstimatedProbability_point3 = 0.0
    addOneEstimatedProbability_point2 = 0.0
    addOneEstimatedProbability_point1 = 0.0
    
    
    if queryTerm not in queryTemrsWithTheirRealFreqIn85KQueriesDict:
        # print queryTerm
        addOneEstimatedProbability_point9 = PARTIAL_CONSTANT_point9 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point9)
        addOneEstimatedProbability_point8 = PARTIAL_CONSTANT_point8 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point8)
        addOneEstimatedProbability_point7 = PARTIAL_CONSTANT_point7 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point7)
        addOneEstimatedProbability_point6 = PARTIAL_CONSTANT_point6 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point6)
        addOneEstimatedProbability_point5 = PARTIAL_CONSTANT_point5 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point5)
        addOneEstimatedProbability_point4 = PARTIAL_CONSTANT_point4 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point4)
        addOneEstimatedProbability_point3 = PARTIAL_CONSTANT_point3 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point3)
        addOneEstimatedProbability_point2 = PARTIAL_CONSTANT_point2 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point2)
        addOneEstimatedProbability_point1 = PARTIAL_CONSTANT_point1 / (NUM_QUERY_TERM_POSITONS_IN_85K_QUERIES + SIZE_OF_VOCABULARY * PARTIAL_CONSTANT_point1)
    else:
        # (realFreq,newFreq_point7,addOneEstimatedProbability_point9,addOneEstimatedProbability_point8,addOneEstimatedProbability_point7,addOneEstimatedProbability_point6,addOneEstimatedProbability_point5,addOneEstimatedProbability_point4,addOneEstimatedProbability_point3,addOneEstimatedProbability_point2,addOneEstimatedProbability_point1)
        (_,_,addOneEstimatedProbability_point9,addOneEstimatedProbability_point8,addOneEstimatedProbability_point7,addOneEstimatedProbability_point6,addOneEstimatedProbability_point5,addOneEstimatedProbability_point4,addOneEstimatedProbability_point3,addOneEstimatedProbability_point2,addOneEstimatedProbability_point1) = queryTemrsWithTheirRealFreqIn85KQueriesDict[queryTerm]
    
    outputFileHanlder.write(line.strip() + " " + str(addOneEstimatedProbability_point9) + " " + str(addOneEstimatedProbability_point8) + " " + str(addOneEstimatedProbability_point7) + " " + str(addOneEstimatedProbability_point6) + " " + str(addOneEstimatedProbability_point5) + " " + str(addOneEstimatedProbability_point4) + " " + str(addOneEstimatedProbability_point3) + " " + str(addOneEstimatedProbability_point2) + " " + str(addOneEstimatedProbability_point1)+ "\n")

inputFileHandler.close()
outputFileHanlder.close()
